%matplotlib inline
%pylab inline


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
import sys
import argparse
import importlib
import facenet
import os
import numpy as np
import h5py
import math
from scipy import misc

import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
tf.logging.set_verbosity(tf.logging.ERROR)

import matplotlib
import matplotlib.patches as mpatches
import matplotlib.pyplot as plt
np.random.seed(None)
pylab.rcParams['figure.figsize'] = (10, 6)

Populating the interactive namespace from numpy and matplotlib

/home/aprml6/.local/lib/python2.7/site-packages/IPython/core/magics/pylab.py:161: UserWarning: pylab import has clobbered these variables: ['f', 'mean', 'std']
`%matplotlib` prevents importing * from pylab and numpy
  "\n`%matplotlib` prevents importing * from pylab and numpy"

img_mean = np.array([134.10714722, 102.52040863, 87.15436554])
img_stddev = np.sqrt(np.array([3941.30175781, 2856.94287109, 2519.35791016]))

latentVarSize = 100
vae_def = importlib.import_module('generative.models.dfc_vae')
vae = vae_def.Vae(latentVarSize)
gen_image_size = vae.get_image_size()

with tf.Graph().as_default():

    images = tf.placeholder(tf.float32, shape=(None, gen_image_size,gen_image_size,3), name='input')

    # Normalize
    images_norm = (images-img_mean) / img_stddev

    # Resize to appropriate size for the encoder 
    images_norm_resize = tf.image.resize_images(images_norm, (gen_image_size,gen_image_size))

    # Create encoder network
    mean, log_variance = vae.encoder(images_norm_resize, True)

    epsilon = tf.random_normal((tf.shape(mean)[0], latentVarSize))
    std = tf.exp(log_variance/2)
    latent_var = mean + epsilon * std

    # Create decoder
    reconstructed_norm = vae.decoder(latent_var, False)

    # Un-normalize
    reconstructed = (reconstructed_norm*img_stddev) + img_mean

    # Create a saver
    saver = tf.train.Saver(tf.trainable_variables(), max_to_keep=3)

    # Start running operations on the Graph
    gpu_memory_fraction = 1.0
    gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=gpu_memory_fraction)
    sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False))
    sess.run(tf.global_variables_initializer())
    sess.run(tf.local_variables_initializer())
    coord = tf.train.Coordinator()
    tf.train.start_queue_runners(coord=coord, sess=sess)

Generate original and target Imgaes and take initial genesÂ¶

with sess.as_default():
    
    # Load checkpoint
    vae_checkpoint = os.path.expanduser('/home/aprml6/kush/ai/facenet/vae/model.ckpt-50000')
    print('Restoring VAE checkpoint: %s' % vae_checkpoint)
    saver.restore(sess, vae_checkpoint)
    
    # Load attribute file
    filename = os.path.expanduser('/home/aprml6/kush/ai/facenet/vae/attribute_vectors.h5')
    with h5py.File(filename,'r') as f:
        latent_vars = np.array(f.get('latent_vars'))
        attributes = np.array(f.get('attributes'))
        #fields = np.array(f.get('fields'))
        attribute_vectors = np.array(f.get('attribute_vectors'))

    attribute_index = 31
    image_indices = [8]
    nrof_images = len(image_indices)
    nrof_interp_steps = 2
    sweep_latent_var = np.zeros((nrof_interp_steps * nrof_images, latentVarSize), np.float32)
    for j in range(nrof_images):
        image_index = image_indices[j]
        idx = np.argwhere(attributes[:, attribute_index] == -1)[image_index, 0]
        initialGenes = latent_vars[idx,:]
        for i in range(nrof_interp_steps):
            sweep_latent_var[i+nrof_interp_steps*j,:] = latent_vars[idx,:] + 5.0*i/nrof_interp_steps*attribute_vectors[attribute_index,:]
            print (sweep_latent_var.shape)
    recon = sess.run(reconstructed, feed_dict={latent_var:sweep_latent_var})

    for i in range(recon.shape[0]):           
        img = facenet.put_images_on_grid(recon[i : i + 1], shape=(1, 1))
        image_filename = os.path.expanduser('/home/aprml6/kush/ai/facenet/vae/add_smile') + str(i) + '.png'
        print('Writing generated image to %s' % image_filename)
        misc.imsave(image_filename, img)

Restoring VAE checkpoint: /home/aprml6/kush/ai/facenet/vae/model.ckpt-50000
(2, 100)
(2, 100)
Writing generated image to /home/aprml6/kush/ai/facenet/vae/add_smile0.png
Writing generated image to /home/aprml6/kush/ai/facenet/vae/add_smile1.png

Original Image and Target ImageÂ¶

originalImage = misc.imread('/home/aprml6/kush/ai/facenet/vae/original.png')
plt.subplot(1, 2, 1)
plt.gca().xaxis.set_visible(False)
plt.gca().yaxis.set_visible(False)
plt.imshow(originalImage)

targetImage = misc.imread('/home/aprml6/kush/ai/facenet/vae/target.png')
plt.subplot(1, 2, 2)
plt.gca().xaxis.set_visible(False)
plt.gca().yaxis.set_visible(False)
plt.imshow(targetImage)

<matplotlib.image.AxesImage at 0x7f23bc678950>

attribute_index = 31
image_index = 10
nrof_images = len(image_indices)
nrof_interp_steps = 1

Genetic learning helpersÂ¶

bestGenes = initialGenes

def evaluateFitness(solution):
    return np.log(np.sum(np.abs(solution - targetImage)))

def generateTwoIndices(n):
    
    """
        Generates two random integers in [0, n) such 
        that first < second
        
    """
    
    first, second = 0, 0
    while first == second:
        first  = int(np.random.uniform() * n)
        second = int(np.random.uniform() * n)
    
    if first > second:
        first, second = second, first

    return first, second

# Load attribute file
filename = os.path.expanduser('/home/aprml6/kush/ai/facenet/vae/attribute_vectors.h5')
with h5py.File(filename,'r') as f:
    latent_vars = np.array(f.get('latent_vars'))
    attributes = np.array(f.get('attributes'))
    attribute_vectors = np.array(f.get('attribute_vectors'))

def crossover(space1, space2):

    first, second = generateTwoIndices(len(space1))
    crossover = np.zeros_like(space1)   
    
    crossover[ : first] = space1[ : first]
    crossover[first : second] = space2[first : second]
    crossover[second : ] = space1[second : ]

    return crossover

def mutate(space):
    
    """
        Introduces mutation in a given solution
    """
    
    randIndex = int(np.random.uniform() * space.shape[0])
    space[randIndex] += (np.random.uniform(-1., 1.))
    return space

def localSearch(latentSpace):
    
    newLatentSpace = [latentSpace]
    
    for i in range(latentSpace.shape[0]):
        tempLatentSpace = latentSpace
        tempLatentSpace[i] += np.random.uniform()
        newLatentSpace.append(tempLatentSpace)

    for i in range(latentSpace.shape[0]):
        tempLatentSpace = latentSpace
        tempLatentSpace[i] -= np.random.uniform()
        newLatentSpace.append(tempLatentSpace)

    newLatentSpace = np.array(newLatentSpace)

    with sess.as_default():
        # Load checkpoint
        vae_checkpoint = os.path.expanduser('/home/aprml6/kush/ai/facenet/vae/model.ckpt-50000')
        saver.restore(sess, vae_checkpoint)
        recon = sess.run(reconstructed, feed_dict={latent_var : newLatentSpace})

    scores = []
    
    # Select top solution based on evaluation function
    for i in range(newLatentSpace.shape[0]):
        img = facenet.put_images_on_grid(recon[i : i + 1], shape=(1, 1))
        misc.imsave('temp.png', img)
        img = misc.imread('temp.png')
        scores.append(evaluateFitness(img))
       
    scores = np.array(scores)
    return newLatentSpace[scores.argsort()[0]]

generationMins = []
generationLatentSpace = []

maxIter = 500
currIter = 0
generationSize = 65
top = 10

latentSpace = np.zeros((generationSize, latentVarSize), np.float32)
for i in range(generationSize):
    latentSpace[i, :] = initialGenes + np.random.uniform(-1., 1., (100,)) * 5

while currIter < maxIter:

    # Generate Generation
    with sess.as_default():
        # Load checkpoint
        vae_checkpoint = os.path.expanduser('/home/aprml6/kush/ai/facenet/vae/model.ckpt-50000')
        saver.restore(sess, vae_checkpoint)
        recon = sess.run(reconstructed, feed_dict={latent_var : latentSpace})

    scores = []
    
    # Select top 10 solutions based on evaluation function
    for i in range(generationSize):
        img = facenet.put_images_on_grid(recon[i : i + 1], shape=(1, 1))
        misc.imsave('temp.png', img)
        img = misc.imread('temp.png')
        scores.append(evaluateFitness(img))
       
    topTenIndices = np.array(scores).argsort()[: top]
    generationMins.append(scores[topTenIndices[0]])
    generationLatentSpace.append(latentSpace[topTenIndices[0] : topTenIndices[0] + 1].flatten().tolist())
    
    print ("Iteration : " + str(currIter) + " Best score : " + str(scores[topTenIndices[0]]))
    for i in range(top):
        img = facenet.put_images_on_grid(recon[topTenIndices[i] : topTenIndices[i] + 1], shape=(1, 1))
        misc.imsave('temp.png', img)
        img = misc.imread('temp.png')
        plt.subplot(1, top, i + 1)
        plt.gca().xaxis.set_visible(False)
        plt.gca().yaxis.set_visible(False)
        plt.imshow(img)
    plt.show()
    
    newLatentSpace = []
    
    for ind in topTenIndices:
        newLatentSpace.append(latentSpace[ind])
    
    for i in range(10):
        for j in range(i, 10):
            newLatentSpace.append(crossover(newLatentSpace[i], newLatentSpace[j]))
    
    for i in range(10, 65):
        if np.random.uniform() > 0.25:
            newLatentSpace[i] = mutate(newLatentSpace[i])
    
    newLatentSpace = np.array(newLatentSpace)
    
    for i in range(65):
        newLatentSpace[i] = localSearch(newLatentSpace[i])

    latentSpace = np.array(newLatentSpace)

    currIter += 1

Iteration : 0 Best score : 14.0693594385

Iteration : 1 Best score : 13.9830898124

Iteration : 2 Best score : 13.8371644119

Iteration : 3 Best score : 13.8152795313

Iteration : 4 Best score : 13.6855134837

Iteration : 5 Best score : 13.6820705897

Iteration : 6 Best score : 13.6672297832

Iteration : 7 Best score : 13.6418616845

Iteration : 8 Best score : 13.6081972939

Iteration : 9 Best score : 13.579697081

plt.plot([i for i in range(len(generationMins))], generationMins)
plt.title('Log MAE loss over iterations')

<matplotlib.text.Text at 0x7f23a5606290>

np.array(generationLatentSpace).shape

(500, 100)

np.argmin(generationMins)

402

Save best imageÂ¶

with sess.as_default():
    # Load checkpoint
    vae_checkpoint = os.path.expanduser('/home/aprml6/kush/ai/facenet/vae/model.ckpt-50000')
    saver.restore(sess, vae_checkpoint)
    recon = sess.run(reconstructed, feed_dict={latent_var : generationLatentSpace[np.argmin(generationMins) : np.argmin(generationMins) + 1]})

img = facenet.put_images_on_grid(recon[0 : 1], shape=(1, 1))
misc.imsave('/home/aprml6/kush/ai/facenet/vae/best.png', img)

originalImage = misc.imread('/home/aprml6/kush/ai/facenet/vae/original.png')
plt.subplot(1, 3, 1)
plt.gca().xaxis.set_visible(False)
plt.gca().yaxis.set_visible(False)
plt.imshow(originalImage)
plt.title('Original Image')

targetImage = misc.imread('/home/aprml6/kush/ai/facenet/vae/target.png')
plt.subplot(1, 3, 2)
plt.gca().xaxis.set_visible(False)
plt.gca().yaxis.set_visible(False)
plt.imshow(targetImage)
plt.title('Target Image')

bestImage = misc.imread('/home/aprml6/kush/ai/facenet/vae/best.png')
plt.subplot(1, 3, 3)
plt.gca().xaxis.set_visible(False)
plt.gca().yaxis.set_visible(False)
plt.imshow(bestImage)
plt.title('Predicted Image')

<matplotlib.text.Text at 0x7f23bc360990>